package io.renren.modules.sys.back.service.impl;

import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.crawler.Page;
import edu.uci.ics.crawler4j.crawler.WebCrawler;
import edu.uci.ics.crawler4j.parser.HtmlParseData;
import edu.uci.ics.crawler4j.url.WebURL;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.select.Elements;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Pattern;


public class SimplewendaCrawlerImpl extends WebCrawler{

    /**
     * 正则表达式匹配指定的后缀文件
     */
    private final static Pattern FILTERS = Pattern.compile(".*(\\.(css|js|gif|jpg" + "|png|mp3|mp3|zip|gz))$");

    /**
     * 这个方法主要是决定哪些url我们需要抓取，返回true表示是我们需要的，返回false表示不是我们需要的Url
     * 第一个参数referringPage封装了当前爬取的页面信息 第二个参数url封装了当前爬取的页面url信息
     * 在这个例子中，我们指定爬虫忽略具有css，js，git，...扩展名的url，只接受以“http://www.ics.uci.edu/”开头的url。
     * 在这种情况下，我们不需要referringPage参数来做出决定。
     */
    @Override
    public boolean shouldVisit(Page referringPage, WebURL url) {
        String href = url.getURL().toLowerCase();
        CrawlController cc= this.getMyController();
        Map<String,Object> para = (Map<String,Object>)cc.getCustomData();
        boolean t = !FILTERS.matcher(href).matches() && href.startsWith(para.get("param").toString());
        para.put("back",false);
        cc.setCustomData(para);
        return t;
    }

    /**
     * This function is called when a page is fetched and ready
     * to be processed by your program.
     */
    @Override
    public void visit(Page page) {

        String url = page.getWebURL().getURL();
       // System.out.println("URL: " + url);
        if (page.getParseData() instanceof HtmlParseData) {
            HtmlParseData htmlParseData = (HtmlParseData) page.getParseData();
            String text = htmlParseData.getText();
            String html = htmlParseData.getHtml();
          //  Set<WebURL> links = htmlParseData.getOutgoingUrls();

           // System.out.println("html;;;;;;;;;;;;;;;;;; " + html);
            CrawlController cc= this.getMyController();
            Map<String,Object> ccd= new HashMap<String,Object>();
            Document doc = Jsoup.parse(html);
            Elements bto = doc.getElementsByTag("h1");
            Elements tagElement = doc.getElementsByClass("info");
            if(bto.size()>0){
                ccd.put("name",bto.get(0).html());
            }
            ccd.put("back",true);
            for(int i=0 ;i<tagElement.size();i++){
                Document subdoc = Jsoup.parse(tagElement.get(i).toString());
                Elements imgs = subdoc.getElementsByTag("img");
                if(imgs.size()>0){
                    for(int j=0 ;j<imgs.size();j++){
                        Element img = imgs.get(j);
                        String imgurl = img.attr("src");
                        String extension = imgurl.substring(imgurl.lastIndexOf('.'));
                        String gurl = imgurl;
                        String hashedName = UUID.randomUUID() + extension;
                        SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");

                        String upload = "D:\\work\\yikuangwiki\\upload" + "/" + sdf.format(new Date());
                        File uploadFile = new File(upload);
                        if (!uploadFile.exists()) {
                            uploadFile.mkdirs();
                        }
                        upload += "/" + hashedName;
                        String webPath = "/upload" + "/" + sdf.format(new Date()) + "/" + hashedName;
                        try {
                            if (getImages(gurl, upload) == true) {
                                tagElement.get(i).getElementsByTag("img").get(j).attr("src",webPath);
                                System.out.println("html;;;;;;;;;;;;;;;;;; " );
                            }
                        } catch (Exception e) {
                            e.printStackTrace();
                        }
                    }
                }
                if(tagElement.get(i).attr("style").equals("background-color: #ebf8ff;")){
                    ccd.put("summer",tagElement.get(i).toString());
                }else{
                    ccd.put("content",tagElement.get(i).toString());
                }
            }
            cc.setCustomData(ccd);
            System.out.println("html;;;;;;;;;;;;;;;;;; " );
        }
    }
    //获取概述
    private Object getwufenlei(String html,String type,String url){
        Document doc = Jsoup.parse(html);
        Element tagElement = doc.getElementById(type);
        if(tagElement ==null){
            return "";
        }
        Elements subimg = tagElement.getElementsByTag("img");
        for(Element element : subimg){
            String imgurl = element.attr("src");
            String extension = imgurl.substring(imgurl.lastIndexOf('.'));
            String gurl = url.split("/")[0]+"//"+url.split("/")[2]+ imgurl;
            String hashedName = UUID.randomUUID() + extension;
            SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");

            String upload = "D:\\work\\yikuangwiki\\upload"+"/" + sdf.format(new Date());
            File uploadFile = new File(upload);
            if (!uploadFile.exists()) {
                uploadFile.mkdirs();
            }
            upload +="/"+hashedName;
            String webPath = "/upload"+"/" + sdf.format(new Date())+"/"+hashedName;
            try {
                if(getImages(gurl,upload)==true){
                    element.attr("src",webPath);
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        Document doc2 = Jsoup.parse(tagElement.toString());
        Elements yao= doc2.getElementsByClass("descript-main");
        return yao.toString();
    }
    //获取概述
    private Object getfenlei(String html,String type,String url){
        Map<String,Object> bakdata = new HashMap<>();
        Document doc = Jsoup.parse(html);
        Element tagElement = doc.getElementById(type);
        if(tagElement ==null){
            return null;
        }
        Document doc2 = Jsoup.parse(tagElement.toString());
        Elements yao= doc2.getElementsByClass("descript-main");
        Document doc3 = Jsoup.parse(yao.toString());
        Elements subfenlei = doc3.getElementsByAttributeValue("style","font-weight:bold;");
        List<String> subname = new ArrayList<>();
        List<String> subcontent = new ArrayList<>();
        if(subfenlei.size()>0) {
            for (int i = 0; i < subfenlei.size(); i++) {
                subname.add(subfenlei.get(i).html());
            }
            //获取分类内容
            if (yao.get(0).childNodes().size() > 0) {
                List<Node> tempnode = yao.get(0).childNodes();
                for (int i = 0; i < tempnode.size(); i++) {
                    Node sb = tempnode.get(i);

                    if (sb.hasAttr("id")) {
                        String scontent = "";
                        for (int j = i + 1; j < tempnode.size(); j++) {
                            Node ssb = tempnode.get(j);
                            if (ssb.hasAttr("id")) break;
                            //有图片这下载图片
                            if (ssb.hasAttr("src")) {
                                String imgurl = ssb.attr("src");
                                String extension = imgurl.substring(imgurl.lastIndexOf('.'));
                                String gurl = url.split("/")[0] + "//" + url.split("/")[2] + imgurl;
                                String hashedName = UUID.randomUUID() + extension;
                                SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");

                                String upload = "D:\\work\\yikuangwiki\\upload" + "/" + sdf.format(new Date());
                                File uploadFile = new File(upload);
                                if (!uploadFile.exists()) {
                                    uploadFile.mkdirs();
                                }
                                upload += "/" + hashedName;
                                String webPath = "/upload" + "/" + sdf.format(new Date()) + "/" + hashedName;
                                try {
                                    if (getImages(gurl, upload) == true) {
                                        ssb.attr("src", webPath);
                                    }
                                } catch (Exception e) {
                                    e.printStackTrace();
                                }
                            }
                            scontent += ssb.toString();
                        }
                        subcontent.add(scontent);
                    }
                }
            }
            bakdata.put("subname",subname);
            bakdata.put("subcontent",subcontent);
        }else{
            bakdata.put("summary",getwufenlei(html,type,url));

        }
        return bakdata;
    }
     /* @param urlPath 图片路径
     * @throws Exception
     */
    public static Boolean getImages(String urlPath,String fileName) throws Exception{
        URL url = new URL(urlPath);//：获取的路径
        //:http协议连接对象
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        conn.setRequestMethod("GET");
        conn.setReadTimeout(6 * 10000);
        if (conn.getResponseCode() <10000){
            InputStream inputStream = conn.getInputStream();
            byte[] data = readStream(inputStream);
            if(data.length>(1024*1)){
                FileOutputStream outputStream = new FileOutputStream(fileName);
                outputStream.write(data);
                System.err.println("第图片下载成功");
                outputStream.close();
                return true;
            }
        }
        return false;
    }

    /**
     * 读取url中数据，并以字节的形式返回
     * @param inputStream
     * @return
     * @throws Exception
     */
    public static byte[] readStream(InputStream inputStream) throws Exception{
        ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
        byte[] buffer = new byte[1024];
        int len = -1;
        while((len = inputStream.read(buffer)) !=-1){
            outputStream.write(buffer, 0, len);
        }
        outputStream.close();
        inputStream.close();
        return outputStream.toByteArray();
    }
}
